{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "height has been deprecated.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ljj/anaconda/envs/python3env/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    " \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.externals import joblib \n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from lightgbm import LGBMRegressor\n",
    "import math\n",
    "%matplotlib inline\n",
    "\n",
    "pd.set_option('display.max_colwidth',1000)\n",
    "pd.set_option('display.height',1000)\n",
    "pd.set_option('display.max_rows',500)\n",
    "pd.set_option('display.max_columns',500)\n",
    "pd.set_option('display.width',1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('./public.train.csv')\n",
    "test_data = pd.read_csv('./public.test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_result = pd.DataFrame()\n",
    "df_result['ID'] = list(test_data['ID'])\n",
    "special_missing_ID = test_data[test_data[(test_data == 0) | (test_data == 0.)].count(axis=1) > 13]['ID']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 异常值处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 负值 -> 正值\n",
    "# power_negative_index = train_data[train_data['发电量'] < 0].index\n",
    "# train_data.loc[power_negative_index, '发电量'] = - train_data.loc[power_negative_index, '发电量']\n",
    "# train_data.loc[power_negative_index, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = pd.concat([train_data, test_data], axis=0).sort_values(by='ID').reset_index().drop(['index'], axis=1)\n",
    "bad_feature = ['ID', '功率A', '功率B', '功率C', '平均功率', '现场温度', '电压A', '电压B', '电压C', '电流B', '电流C', '转换效率', '转换效率A', '转换效率B', '转换效率C']\n",
    "bad_index = all_data[bad_feature][\n",
    "    (all_data[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std()) | \n",
    "    (all_data[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "].dropna(how='all').index\n",
    "\n",
    "nn_bad_data = all_data.loc[np.concatenate([bad_index - 1, bad_index, bad_index + 1])].sort_values(by='ID', ascending=True).drop_duplicates()\n",
    "bad_data = all_data.loc[bad_index].sort_values(by='ID', ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bad_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上下记录均值替代异常值\n",
    "for idx, line in bad_data.iterrows():\n",
    "    ID = line['ID']\n",
    "    col_index = line[bad_feature][ \n",
    "        (line[bad_feature] > all_data[bad_feature].mean() + 2 * all_data[bad_feature].std())| \n",
    "        (line[bad_feature] < all_data[bad_feature].mean() - 2 * all_data[bad_feature].std())\n",
    "    ].index\n",
    "    index = all_data[all_data['ID'] == ID].index\n",
    "    \n",
    "    before_offset = 1\n",
    "    while (idx + before_offset)in bad_index:\n",
    "        before_offset += 1\n",
    "\n",
    "    after_offset = 1\n",
    "    while (idx + after_offset) in bad_index:\n",
    "        after_offset += 1\n",
    "    \n",
    "    replace_value = (all_data.loc[index - before_offset, col_index].values + all_data.loc[index + after_offset, col_index].values) / 2\n",
    "    all_data.loc[index, col_index] = replace_value[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 拆分数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8409"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data = all_data[all_data['ID'].isin(df_result['ID'])].drop(['发电量'], axis=1).reset_index().drop(['index'], axis=1)\n",
    "len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = all_data.drop(all_data[all_data['ID'].isin(df_result['ID'])].index).reset_index().drop(['index'], axis=1)\n",
    "len(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 去除重复值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data.drop_duplicates(train_data.columns.drop('ID'), keep='first', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train_data['ID'] = train_data['ID'].apply(lambda x: x % 190)\n",
    "# forest_imp = RandomForestRegressor(n_estimators=150, max_features='log2', random_state=2, n_jobs=8)\n",
    "# X = train_data.drop(['发电量'], axis=1)\n",
    "# y = train_data['发电量']\n",
    "# forest_imp.fit(X, y)\n",
    "# importance = forest_imp.feature_importances_\n",
    "\n",
    "# indices = np.argsort(importance)[:: -1]\n",
    "# print(indices, indices.shape)\n",
    "\n",
    "# for f in range(X.shape[1]):\n",
    "#     print(\"%2d) %-*s %f\" %\n",
    "#           (f + 1, 30, train_data.drop(['发电量'], axis=1).columns[indices[f]], importance[indices[f]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model: Xgboost, Sklearn_GBDT, RandomForest, LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_train_data(train_data, test_data, poly=False, select=False):\n",
    "    # 1. ID % 190\n",
    "#     train_data['ID'] = train_data['ID'].apply(lambda x: x % 190)\n",
    "#     sub_data = test_data\n",
    "#     sub_data['ID'].apply(lambda x: x % 190)\n",
    "#     y = train_data['发电量']\n",
    "#     X = train_data.drop(['发电量'], axis=1)\n",
    "    \n",
    "    # 2. ID\n",
    "#     y = train_data['发电量']\n",
    "#     X = train_data.drop(['发电量'], axis=1)\n",
    "#     sub_data = test_data\n",
    "    \n",
    "    # 3. without ID\n",
    "    y = train_data['发电量']\n",
    "    X = train_data.drop(['发电量','ID'], axis=1)\n",
    "    sub_data = test_data.drop(['ID'], axis=1)\n",
    "    \n",
    "    # 4. shuffle\n",
    "#     X, y = shuffle(X, y, random_state=123)\n",
    "#     X, y = shuffle(X, y, random_state=1234)\n",
    "\n",
    "    if poly:\n",
    "        from sklearn.preprocessing import PolynomialFeatures\n",
    "        poly = PolynomialFeatures(degree=2, interaction_only=True)\n",
    "        X = poly.fit_transform(X)\n",
    "        sub_data = poly.transform(sub_data)\n",
    "        \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n",
    "\n",
    "    if select:\n",
    "        from sklearn.feature_selection import SelectFromModel\n",
    "        sm = SelectFromModel(GradientBoostingRegressor(random_state=2))\n",
    "        X_train = sm.fit_transform(X_train, y_train)\n",
    "        X_test = sm.transform(X_test)\n",
    "        sub_data = sm.transform(sub_data)\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test, sub_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cal_score(mse):\n",
    "    return 1 / (1 + math.sqrt(mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7134, 54) (8409, 54)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test, sub_data = generate_train_data(train_data, test_data, poly=True, select=True)\n",
    "print(X_train.shape, sub_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgbt = xgb.XGBRegressor(n_estimators=300, max_depth=3, random_state=2, n_jobs=8)\n",
    "gbdt = GradientBoostingRegressor(n_estimators=300, max_depth=3, max_features='log2', random_state=2)\n",
    "forest = RandomForestRegressor(n_estimators=100, max_features='log2', random_state=2, n_jobs=8)\n",
    "\n",
    "lgb_params = {}\n",
    "lgb_params['n_estimators'] = 300\n",
    "lgb_params['max_depth'] = 3 \n",
    "lgb_params['random_state'] = 2\n",
    "lgb = LGBMRegressor(**lgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(X_train, y_train):\n",
    "    xgbt.fit(X_train, y_train)\n",
    "    gbdt.fit(X_train, y_train)\n",
    "    forest.fit(X_train, y_train)\n",
    "    lgb.fit(X_train, y_train)\n",
    "\n",
    "def predict(X_test, y_test):\n",
    "    y_pred_xgb = xgbt.predict(X_test)\n",
    "    mse_xgb = mean_squared_error(y_test.values, y_pred_xgb)\n",
    "    \n",
    "    y_pred_gbdt = gbdt.predict(X_test)\n",
    "    mse_gbdt = mean_squared_error(y_test.values, y_pred_gbdt)\n",
    "    \n",
    "    y_pred_forest = forest.predict(X_test)\n",
    "    mse_forest = mean_squared_error(y_true=y_test, y_pred=y_pred_forest)\n",
    "    \n",
    "    y_pred_lgb = lgb.predict(X_test)\n",
    "    mse_lgb = mean_squared_error(y_true=y_test, y_pred=y_pred_lgb)\n",
    "    \n",
    "    res = pd.DataFrame()\n",
    "    res['model'] = np.array(['XGBoost', 'Sklearn_GBDT', 'RandomForest', 'LightGBM'])\n",
    "    res['mse'] = np.array([mse_xgb, mse_gbdt, mse_forest, mse_lgb])\n",
    "    res['score'] = np.array([cal_score(mse_xgb), cal_score(mse_gbdt), cal_score(mse_forest), cal_score(mse_lgb)])\n",
    "    return res\n",
    "\n",
    "def cross_validation_using_mse(X_train, y_train, cv=5):\n",
    "    scores_xgb = cross_val_score(xgbt, X_train, y_train, cv=cv, scoring='neg_mean_squared_error')\n",
    "    xgb_avg = np.average(-scores_xgb)\n",
    "    print('Average XGB - MSE:', xgb_avg, ' - Score:', cal_score(xgb_avg))\n",
    "    \n",
    "    scores_gbdt = cross_val_score(gbdt, X_train, y_train, cv=cv, scoring='neg_mean_squared_error')\n",
    "    gbdt_avg = np.average(-scores_gbdt)\n",
    "    print('Average GBDT - MSE:', gbdt_avg, ' - Score:', cal_score(gbdt_avg))\n",
    "    \n",
    "    scores_forest = cross_val_score(forest, X_train, y_train, cv=cv, scoring='neg_mean_squared_error')\n",
    "    rf_avg = np.average(-scores_forest)\n",
    "    print('Average RF - MSE:', rf_avg, ' - Score:', cal_score(rf_avg))\n",
    "    \n",
    "    scores_lgb = cross_val_score(lgb, X_train, y_train, cv=cv, scoring='neg_mean_squared_error')\n",
    "    lgb_avg = np.average(-scores_lgb)\n",
    "    print('Average LGB - MSE:', lgb_avg, ' - Score:', cal_score(lgb_avg))\n",
    "    \n",
    "    res = pd.DataFrame({\n",
    "        'XGBoost': -scores_xgb,\n",
    "        'Sklearn_GBDT': -scores_gbdt,\n",
    "        'RandomForest': -scores_forest,\n",
    "        'LightGBM': -scores_lgb\n",
    "    })\n",
    "    \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mse</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XGBoost</td>\n",
       "      <td>0.016908</td>\n",
       "      <td>0.884931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sklearn_GBDT</td>\n",
       "      <td>0.020530</td>\n",
       "      <td>0.874675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RandomForest</td>\n",
       "      <td>0.016276</td>\n",
       "      <td>0.886858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LightGBM</td>\n",
       "      <td>0.016375</td>\n",
       "      <td>0.886551</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          model       mse     score\n",
       "0       XGBoost  0.016908  0.884931\n",
       "1  Sklearn_GBDT  0.020530  0.874675\n",
       "2  RandomForest  0.016276  0.886858\n",
       "3      LightGBM  0.016375  0.886551"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(X_train, y_train)\n",
    "predict(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average XGB - MSE: 0.03248307780460091  - Score: 0.8472920289633665\n",
      "Average GBDT - MSE: 0.033259116671348274  - Score: 0.8457583538929327\n",
      "Average RF - MSE: 0.030792564684868108  - Score: 0.8507176759148936\n",
      "Average LGB - MSE: 0.030453712226382056  - Score: 0.8514189474729728\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LightGBM</th>\n",
       "      <th>RandomForest</th>\n",
       "      <th>Sklearn_GBDT</th>\n",
       "      <th>XGBoost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.013062</td>\n",
       "      <td>0.013684</td>\n",
       "      <td>0.016057</td>\n",
       "      <td>0.013085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.023412</td>\n",
       "      <td>0.021713</td>\n",
       "      <td>0.025226</td>\n",
       "      <td>0.029165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.013821</td>\n",
       "      <td>0.014352</td>\n",
       "      <td>0.017467</td>\n",
       "      <td>0.016010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.085465</td>\n",
       "      <td>0.087725</td>\n",
       "      <td>0.087286</td>\n",
       "      <td>0.087944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.016508</td>\n",
       "      <td>0.016488</td>\n",
       "      <td>0.020259</td>\n",
       "      <td>0.016211</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LightGBM  RandomForest  Sklearn_GBDT   XGBoost\n",
       "0  0.013062      0.013684      0.016057  0.013085\n",
       "1  0.023412      0.021713      0.025226  0.029165\n",
       "2  0.013821      0.014352      0.017467  0.016010\n",
       "3  0.085465      0.087725      0.087286  0.087944\n",
       "4  0.016508      0.016488      0.020259  0.016211"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation_using_mse(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]), cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgbt2 = xgb.XGBRegressor(n_estimators=300, max_depth=3, random_state=2, n_jobs=8)\n",
    "gbdt2 = GradientBoostingRegressor(n_estimators=300, max_depth=3, max_features='log2', random_state=2)\n",
    "forest2 = RandomForestRegressor(n_estimators=100, max_features='log2', random_state=2, n_jobs=8)\n",
    "lgb2 = LGBMRegressor(**lgb_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMRegressor(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "       importance_type='split', learning_rate=0.1, max_depth=3,\n",
       "       min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "       n_estimators=300, n_jobs=-1, num_leaves=31, objective=None,\n",
       "       random_state=2, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "       subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbt2.fit(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]))\n",
    "gbdt2.fit(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]))\n",
    "forest2.fit(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]))\n",
    "lgb2.fit(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "          metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "          weights='uniform')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "knn = KNeighborsRegressor(n_neighbors=5)\n",
    "knn.fit(np.concatenate([X_train, X_test]), np.concatenate([y_train, y_test]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0210029  0.03720661 0.02300897 0.07728633 0.07190166]\n",
      "Average XGB - MSE: 0.04608129499247423  - Score: 0.823271892786727\n"
     ]
    }
   ],
   "source": [
    "scores_knn = cross_val_score(knn, X_train, y_train, cv=5, scoring='neg_mean_squared_error')\n",
    "print(-scores_knn)\n",
    "knn_avg = np.average(-scores_knn)\n",
    "print('Average XGB - MSE:', knn_avg, ' - Score:', cal_score(knn_avg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_X_train = np.concatenate([X_train, X_test])\n",
    "all_y_train = np.concatenate([y_train, y_test])\n",
    "regrs = [xgbt2, gbdt2, forest2, lgb2, knn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.00169305 0.00289093 0.00168124 0.01063128 0.00199901]\n",
      "Average XGB - MSE: 0.0037791044520296226  - Score: 0.9420858195951028\n"
     ]
    }
   ],
   "source": [
    "stack_X_train = np.zeros((all_X_train.shape[0],len(regrs)))\n",
    "stack_X_sub = np.zeros((sub_data.shape[0],len(regrs)))\n",
    "\n",
    "for j, regr in enumerate(regrs):\n",
    "    stack_X_train[:, j] = regr.predict(all_X_train)\n",
    "    stack_X_sub[:, j] = regr.predict(sub_data)\n",
    "\n",
    "stacker = RandomForestRegressor(n_estimators=70, random_state=2)\n",
    "mse_stack = cross_val_score(stacker, stack_X_train, all_y_train, cv=5, scoring='neg_mean_squared_error')\n",
    "stack_avg = np.average(-mse_stack)\n",
    "print(-mse_stack)\n",
    "print('Average XGB - MSE:', stack_avg, ' - Score:', cal_score(stack_avg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=70, n_jobs=1,\n",
       "           oob_score=False, random_state=2, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stacker.fit(stack_X_train, all_y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def output(sub_data, model='gbdt'):\n",
    "    if model == 'nn':\n",
    "        sub_data_nn = min_max_scaler.transform(np.array(sub_data))\n",
    "        sub_data_nn = poly.transform(sub_data_nn)\n",
    "        pred = NN.predict(sub_data_nn)\n",
    "\n",
    "    elif model == 'gbdt':\n",
    "        pred = gbdt2.predict(sub_data)\n",
    "\n",
    "    elif model == 'xgb':\n",
    "        pred = xgbt2.predict(sub_data)\n",
    "\n",
    "    elif model == 'rf':\n",
    "        pred = forest2.predict(sub_data)\n",
    "    \n",
    "    elif model == 'lgb':\n",
    "        pred = lgb2.predict(sub_data)\n",
    "        \n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pred_stack = stacker.predict(stack_X_sub)\n",
    "# df_result['score'] = pred_stack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result['score'] = output(sub_data, 'rf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "index = df_result[df_result['ID'].isin(special_missing_ID)].index\n",
    "df_result.loc[index, 'score'] = 0.379993053\n",
    "df_result[df_result['ID'].isin(special_missing_ID)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_result.to_csv('submit_rf_poly_select_dropdup_replaceoutlier_nega20.csv', index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
